﻿using HtmlAgilityPack;
using ScrapySharp.Extensions;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace MJTop.FZhan
{
    /// <summary>
    /// 仿站配置
    /// </summary>
    public class Crawler
    {
        /// <summary>
        /// 默认请求的Url文件名
        /// </summary>
        public  const string DefaultHtmlName = "index.html";

        public Crawler()
        {

        }

        public Crawler(string url)
           : this(new Uri(url), Encoding.UTF8)
        {

        }

        public Crawler(string url, Encoding charset)
            : this(new Uri(url), charset)
        {

        }

        public Crawler(Uri uri)
             : this(uri, Encoding.UTF8)
        {

        }

        public Crawler(Uri uri, Encoding charset)
        {
            this.Uri = uri;
            this.Charset = charset;
            this.SaveDir = Uri.LocalAbsDir();
        }

        /// <summary>
        /// 请求地址URI
        /// </summary>
        public Uri Uri { get; }

        /// <summary>
        /// 请求地址
        /// </summary>
        public string Url
        {
            get
            {
                return Uri.AbsoluteUri;
            }
        }

        /// <summary>
        /// 网页编码,默认utf-8
        /// </summary>
        public Encoding Charset { get; }

        /// <summary>
        /// 仿站文件保存路径：默认桌面上的  FZhan 文件夹
        /// </summary>
        public string SaveDir { get; }


        /// <summary>
        /// 首页内要请求的路径文件的扩展名：如 asp apsx ashx jsp php 
        /// </summary>
        public List<string> ExtNames { get; set; } = new List<string>()
        {
            ".asp",
            //".aspx",
            //".ashx",
            //".jsp",
            //".php"
        };

        #region 静态属性

        /// <summary>
        /// 最大仿站深度:2级
        /// </summary>
        //public int MaxDeep { get; set; } = 2;

        /// <summary>
        /// 当前仿站深度
        /// </summary>
        //public int CurrDeep { get; set; } = 0;

        
        /// <summary>
        ///静态属性 下载历史的列表：如果存在，则表示已经下载过，防止重复下载
        /// </summary>
        public static ConcurrentBag<string> DownLoad_History
        {
            get;
            private set;
        } = new ConcurrentBag<string>(); 

        #endregion


        //public string ParentHtmlPath { get; set; }


        //public string ParentHtml
        //{
        //    get;set;
        //}


        /// <summary>
        /// 通过url请求地址得出 默认首页
        /// </summary>
        public string FileName
        {
            get
            {
                string fileName = this.Uri.ReqFile();

                if (string.IsNullOrWhiteSpace(fileName))
                {
                    return DefaultHtmlName;
                }

                string extName = Path.GetExtension(fileName);

                if (string.IsNullOrWhiteSpace(extName))
                {
                    fileName = fileName + ".html";
                }
                else if (ExtNames.Contains(extName, StringComparer.OrdinalIgnoreCase))
                {
                    fileName = fileName.Replace(extName, "") + ".html";
                }
                return fileName;
            }
        }

        /// <summary>
        /// html文件 磁盘绝对路径
        /// </summary>
        public string FilePath
        {
            get
            {
                string filePath = Path.Combine(this.SaveDir, this.FileName);
                return filePath;
            }
        }

        /// <summary>
        ///请求方式： 默认 GET
        /// </summary>
        public ReqMethod Method { get; private set; } = ReqMethod.GET;

        /// <summary>
        /// 特殊情形下： 需指定cookie 才能请求成功
        /// </summary>
        public string Cookie { get; set; }

        /// <summary>
        /// 特殊情形下： 需指定Host 才能请求成功
        /// </summary>
        public string Host { get; set; }

        /// <summary>
        /// 特殊情形下： 需指定Referer 才能请求成功
        /// </summary>
        public string Referer { get; set; }

        /// <summary>
        /// 指定请求头，默认情况下不需要指定
        /// </summary>
        public NameValueCollection Header { get; set; }

        /// <summary>
        /// Post方式时，需提交的数据,只有第一次发起请求时才使用
        /// </summary>
        //public NameValueCollection PostValues { get; set; }


        /// <summary>
        /// 当前页面的Html内容
        /// </summary>
        public string Html { get; private set; }

        /// <summary>
        /// 当前页面的Html跟节点
        /// </summary>
        public HtmlNode RootNode
        {
            get
            {
                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(this.Html);
                return doc.DocumentNode;
            }
        }

        /// <summary>
        /// 自定义 要下载的资源
        /// Key： jquery 方式定位的 标签
        /// Values ： 指定一个或多个 属性名称
        /// </summary>
        public ExpressionAttrs DownAttrs { get; set; }

        /// <summary>
        /// 保存请求的url返回的html
        /// </summary>
        private string GetHtml()
        {
            WebClient client = new WebClient()
            {
                Encoding = this.Charset
            };

            string html = null;

            try
            {
                #region 设置请求头
                client.Headers.Add(HttpRequestHeader.Accept, "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*; q = 0.8");
                client.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate, sdch");
                client.Headers.Add(HttpRequestHeader.AcceptLanguage, "zh-CN,zh;q=0.8");
                client.Headers.Add(HttpRequestHeader.CacheControl, "max-age=0");

                if (!string.IsNullOrWhiteSpace(this.Cookie))
                {
                    client.Headers.Add(HttpRequestHeader.Cookie, this.Cookie);
                }

                if (!string.IsNullOrWhiteSpace(this.Host))
                {
                    client.Headers.Add(HttpRequestHeader.Host, this.Host);
                }

                if (!string.IsNullOrWhiteSpace(this.Referer))
                {
                    client.Headers.Add(HttpRequestHeader.Referer, this.Referer);
                }

                client.Headers.Add(HttpRequestHeader.Upgrade, "1");
                client.Headers.Add(HttpRequestHeader.UserAgent, "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36");

                if (this.Header != null && this.Header.Count > 0)
                {
                    foreach (string key in this.Header.AllKeys)
                    {
                        client.Headers[key] = this.Header[key];
                    }
                } 
                #endregion

                if (this.Method == ReqMethod.GET)
                {
                    html = client.DownloadString(this.Url);
                }
                //else
                //{
                //    this.PostValues = this.PostValues ?? new NameValueCollection();
                //    html = this.Charset.GetString(client.UploadValues(this.Url, this.PostValues));
                //}
            }
            catch (Exception ex)
            {
                throw ex;
            }
            finally
            {
                client.Dispose();
            }
            return html;
        }


        private bool ExistDownload(string relativePath)
        {
            return DownLoad_History.Contains(relativePath.Trim('/', '.'), StringComparer.OrdinalIgnoreCase);
        }

        private void AddHistory(string relativePath)
        {
            DownLoad_History.Add(relativePath.Trim('/', '.'));
        }


        /// <summary>
        /// 
        /// </summary>
        /// <param name="extNames">动态网站开发技术页面的扩展名：如 aspx aspx jsp php </param>
        public void Start(bool isContinue = true)
        {
            try
            {
                Console.WriteLine("开始：" + this.Url);
                this.Html = GetHtml();
            }
            catch (Exception ex)
            {
                Console.WriteLine("请求失败：" + this.Url);
                Console.WriteLine(ex.ToString());
                Console.WriteLine("------------------------------------------------------");
                return;
            }
           

            List<Exception> lstExp = new List<Exception>();
            List<string> lstPath = new List<string>();
            
            #region 通过修改 HtmlNode 改变 Html的值

            var rootNode = this.RootNode;

            //处理页面中的 Css样式文件
            var linkNodes = rootNode.SelectNodes("//link");
            var cssFiles = DownLoadRes(linkNodes, ResourcesType.href, ref lstExp);

            //处理页面中的 Js文件
            var scriptNodes = rootNode.SelectNodes("//script");
            var scriptFiles = DownLoadRes(scriptNodes, ResourcesType.src, ref lstExp);

            //处理页面中的 Img文件
            var imgNodes = rootNode.SelectNodes("//img");
            var imgFiles = DownLoadRes(imgNodes, ResourcesType.src, ref lstExp);

            //重新赋值修改后的html
            this.Html = rootNode.InnerHtml;

            #endregion


            //直接通过处理 Html， 处理页面中的 背景图片或字体
            this.Html = BgUrlSearchHandle(this.Url,this.Html, ref lstExp);

            if (cssFiles.Any())
            {
                foreach (var item in cssFiles)
                {                   
                    string content = File.ReadAllText(item.Value, this.Charset);
                    //处理url文件中包含 资源，如：图片或字体
                    string newContent = BgUrlSearchHandle(item.Key, content, ref lstExp);
                    if (content.Equals(newContent))
                    {
                        File.WriteAllText(item.Value, newContent, this.Charset);
                    }

                    //处理import 导入 css文件的情况
                    var importCssFiles = ImportCssSearchHandle(item.Key, content, ref lstExp);
                    //todo：可能还要深度的 对导入的css文件解析下载 bgurl 资源 ，这里 不再做处理。
                }
            }

            //处理 自定义选择器 对应的 属性/属性值
            if (this.DownAttrs != null && this.DownAttrs.Count > 0)
            {
                DownLoadCustomize(this.DownAttrs, ref lstExp);
            }

            string htmlContent = this.Html;

            //递归处理。。。
           
            if (isContinue && ExtNames.Any())
            {
                List<Task> lstTask = new List<Task>();
                foreach (var extName in ExtNames)
                {
                    string reg = string.Format(@"(.+['""])(.+\.{0}(\?.+?)?)(['""].+)", extName.TrimStart('.'));

                    var mats = Regex.Matches(this.Html, reg, RegexOptions.Compiled | RegexOptions.IgnoreCase);
                    
                    foreach (Match mat in mats)
                    {
                        string reqFileName = mat.Groups[2].Value;
                        Uri reqUri = this.Uri.CombineUrl(reqFileName);

                        string htmlUrl = reqUri.AbsolutePath.Replace(extName, ".html");

                        htmlContent = htmlContent.Replace(reqFileName, htmlUrl);

                        var tsk = Task.Run((Action)(() =>
                        {
                            Crawler cra = Clone(reqUri);
                            cra.Start(false);
                        }));
                        
                        lstTask.Add(tsk);

                        if (lstTask.Count(t => t.Status != TaskStatus.RanToCompletion) > (14))
                        {
                            Task.WaitAny(lstTask.ToArray());
                            lstTask = lstTask.Where(t => t.Status != TaskStatus.RanToCompletion).ToList();
                        }
                    }
                    Task.WaitAll(lstTask.ToArray());
                }
            }

            //前面都处理完成后，将html写入至文件中。
            File.WriteAllText(this.FilePath, htmlContent, this.Charset);

            Console.WriteLine("结束：" + this.Url);
        }

        private Crawler Clone(string url)
        {
            return Clone(new Uri(url));
        }

        private  Crawler Clone(Uri uri)
        {
            Crawler cra = new Crawler(uri);
            cra.Cookie = this.Cookie;
            cra.Host = this.Host;
            cra.Referer = this.Referer;
            cra.Header = this.Header;
            return cra;
        }
        


        /// <summary>
        /// css文件内部 import其他的样式文件
        /// </summary>
        /// <param name="reqUrl"></param>
        /// <param name="content"></param>
        /// <param name="lstExp"></param>
        /// <returns></returns>
        private Dictionary<string,string> ImportCssSearchHandle(string url, string content, ref List<Exception> lstExp)
        {
            //处理css文件中导入其他css文件的情况，返回 css文件名称
            /*
             * 
                @import "font-awesome.min.css";
                @import url("print.css") print;
                @import "common.css" screen, projection;
                @import url('landscape.css') screen and (orientation:landscape);
                @import url('mobile.css') (max-width: 680px);
             * 
             */

            Dictionary<string, string> dict = new Dictionary<string, string>();
            WebClient wc = new WebClient()
            {
                Encoding = this.Charset
            };

            List<string> lstImportName = new List<string>();

            string reg = @".*?@import\s+(url\()?['""](.+\.css)\)?['""].+";

            try
            {
                var mats = Regex.Matches(content, reg, RegexOptions.Compiled | RegexOptions.IgnoreCase);

                foreach (Match mat in mats)
                {
                    string reqFileName = mat.Groups[2].Value;

                    //不包含则下载
                    if (!ExistDownload(reqFileName))
                    {
                        Uri re_Uri = new Uri(url).CombineUrl(reqFileName);

                        string reqUrl = re_Uri.AbsoluteUri;

                        string res_Path = re_Uri.LocalAbsPath();

                        //修改href src 关联的路径
                        //content = content.Replace(reqFileName, reqFileName.TrimStart('/', '.'));

                        try
                        {
                            wc.DownloadFile(reqUrl, res_Path);
                            dict.Add(reqUrl, res_Path);

                            //存入下载历史表中
                            AddHistory(reqFileName);
                        }
                        catch (Exception ex)
                        {
                            lstExp.Add(ex);
                            LogUtils.LogError("执行出错", Developer.MJ, ex, reqUrl, res_Path);
                        }
                    }
                    else
                    {

                    }
                }
            }
            catch(Exception ex)
            {
                lstExp.Add(ex);
                LogUtils.LogError("执行出错", Developer.MJ, ex, this);
            }
            finally
            {
                wc.Dispose();
            }
            return dict;
        }

        /// <summary>
        /// 处理页面中的 背景资源
        /// </summary>
        /// <param name="lstExp"></param>
        private string BgUrlSearchHandle(string url, string content ,ref List<Exception> lstExp)
        {
            WebClient wc = new WebClient()
            {
                Encoding = this.Charset
            };

            string reg = @"url\((['""])?(.+?)(['""])?\)";
            try
            {
                var mats = Regex.Matches(content, reg, RegexOptions.Compiled | RegexOptions.IgnoreCase);
                foreach (Match mat in mats)
                {
                    string reqFileName = mat.Groups[2].Value;

                    //不包含则下载
                    if (!ExistDownload(reqFileName))
                    {
                        Uri re_Uri = new Uri(url).CombineUrl(reqFileName);

                        string reqUrl = re_Uri.AbsoluteUri;

                        string res_Path = re_Uri.LocalAbsPath();

                        //修改href src 关联的路径
                        //content = content.Replace(reqFileName, reqFileName.TrimStart('/', '.'));

                        try
                        {
                            wc.DownloadFile(reqUrl, res_Path);

                            //存入下载历史表中
                            AddHistory(reqFileName);
                        }
                        catch (Exception ex)
                        {
                            lstExp.Add(ex);
                            LogUtils.LogError("执行出错", Developer.MJ, ex, reqUrl, res_Path);
                        }
                    }
                    else
                    {

                    }
                }
            }
            catch (Exception ex)
            {
                lstExp.Add(ex);
                LogUtils.LogError("执行出错", Developer.MJ, ex, this);
            }
            finally
            {
                wc.Dispose();
            }
            return content;
        }
        
        /// <summary>
        /// 下载资源文件
        /// 样式:href
        /// 脚本:src
        /// 图片:src
        /// </summary>
        /// <param name="nodes"></param>
        /// <param name="type"></param>
        /// <param name="lstExp"></param>
        private Dictionary<string,string> DownLoadRes(HtmlNodeCollection nodes, ResourcesType type, ref List<Exception> lstExp)
        {
            if (nodes == null || !nodes.Any())
            {
                return new Dictionary<string, string>();
            }

            WebClient wc = new WebClient()
            {
                Encoding = this.Charset
            };
            Dictionary<string,string> dictPath = new Dictionary<string, string>();
            try
            {
                foreach (var node in nodes)
                {
                    var attr = node.Attributes[type.ToString()];

                    if (attr != null && !string.IsNullOrWhiteSpace(attr.Value))
                    {
                        //不包含则下载
                        if (!ExistDownload(attr.Value))
                        {
                            string attrValue = attr.Value;

                            Uri re_Uri = this.Uri.CombineUrl(attrValue);

                            string reqUrl = re_Uri.AbsoluteUri;

                            string res_Path = re_Uri.LocalAbsPath();

                            //修改href src 关联的路径
                            //attr.Value = attrValue.TrimStart('/');

                            try
                            {
                                wc.DownloadFile(reqUrl, res_Path);

                                if (type == ResourcesType.href)//样式，不是 Favicon.ico 文件
                                {
                                    if (Path.GetExtension(res_Path).Equals(".css", StringComparison.OrdinalIgnoreCase))
                                    {
                                        dictPath.Add(reqUrl, res_Path);
                                    }
                                }
                                else
                                {
                                    dictPath.Add(reqUrl, res_Path);
                                }

                                //存入下载历史表中
                                AddHistory(attr.Value);
                            }
                            catch (Exception ex)
                            {
                                lstExp.Add(ex);
                                LogUtils.LogError("执行出错", Developer.MJ, ex, reqUrl, res_Path);
                            }
                        }
                        else
                        {

                        }
                    }
                }
            }
            catch (Exception ex)
            {
                lstExp.Add(ex);
                LogUtils.LogError("执行出错", Developer.MJ, ex, this);
            }
            finally
            {
                wc.Dispose();
            }
            return dictPath;
        }


        /// <summary>
        /// 下载自定义资源
        /// </summary>
        /// <param name="nvc"></param>
        /// <param name="lstExp"></param>
        private void DownLoadCustomize(ExpressionAttrs nvc, ref List<Exception> lstExp)
        {
            var rootNode = this.RootNode;
            if (nvc != null && nvc.Count > 0)
            {
                WebClient wc = new WebClient()
                {
                    Encoding = this.Charset
                };

                try
                {
                    foreach (var key in nvc.AllKeys)
                    {
                        var nodes = rootNode.CssSelect(key);
                        if (nodes != null && nodes.Any())
                        {
                            string[] values = nvc.GetValues(key);
                            foreach (HtmlNode node in nodes)
                            {
                                foreach (string attrName in values)
                                {
                                    var attr = node.Attributes[attrName];
                                    if (attr != null && !string.IsNullOrWhiteSpace(attr.Value))
                                    {
                                        //不包含则下载
                                        if (!ExistDownload(attr.Value))
                                        {
                                            //处理下载 标签的 自定义属性值 的路径

                                            string attrValue = attr.Value;

                                            Uri re_Uri = this.Uri.CombineUrl(attrValue);

                                            string reqUrl = re_Uri.AbsoluteUri;

                                            string res_Path = re_Uri.LocalAbsPath();

                                            //修改href src 关联的路径
                                            //attr.Value = attrValue.TrimStart('/');


                                            try
                                            {
                                                wc.DownloadFile(reqUrl, res_Path);

                                                //存入下载历史表中
                                                AddHistory(attr.Value);
                                            }
                                            catch (Exception ex)
                                            {
                                                lstExp.Add(ex);
                                                LogUtils.LogError("执行出错", Developer.MJ, ex, reqUrl, res_Path);
                                            }
                                        }
                                        else
                                        {

                                        }
                                    }
                                }
                            }
                        }
                    }
                }
                catch (Exception ex)
                {
                    lstExp.Add(ex);
                    LogUtils.LogError("执行出错", Developer.MJ, ex, this);
                }
                finally
                {
                    wc.Dispose();
                }
            }
            //注意：修改后 需要重新 赋值给 Html属性
            this.Html = rootNode.InnerHtml;
        }


    }
}
